{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "9e71a8c7",
   "metadata": {},
   "source": [
    "在双模型的基础上,进一步缓解自举.\n",
    "\n",
    "在计算target时,使用原模型计算动作,使用延迟模型计算target,而不是直接取max."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "91162df8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAASuklEQVR4nO3dfUxTZ98H8C8VWuXllKGjHZFGE90c8W1DxaN5ni3aG6bEzMmSbTGOGaPRFSOymIzEl+i2YFwynZviP5u6LM6FJW6R+DJSFbNYRXE8QVTueccNorad+nAKKC3Q6/lj4TyroqNQelH9fpKTeM71a/s7l/ab9lwcjBNCCBARSWCQ3QARPb0YQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJI20ANq1axfGjBmD4cOHIycnBzU1NbJaISJJpATQ999/j5KSEmzatAkXL17ElClTkJeXB6/XK6MdIpIkTsbNqDk5OZg+fTq+/PJLAEAwGERmZiZWr16NDz/8MNrtEJEk8dF+wUAggNraWpSWlurHDAYD7HY7XC5Xr4/x+/3w+/36fjAYxN27dzFy5EjExcUNes9EFB4hBFpbW5GRkQGD4dFftKIeQLdv30Z3dzcsFkvIcYvFgqtXr/b6mLKyMmzevDka7RFRBDU3N2P06NGPHI96APVHaWkpSkpK9H1N02Cz2dDc3AxFUSR2RkS98fl8yMzMREpKymProh5Ao0aNwrBhw+DxeEKOezweWK3WXh9jMplgMpkeOq4oCgOIaAj7p0skUV8FMxqNyM7OhtPp1I8Fg0E4nU6oqhrtdohIIilfwUpKSlBYWIhp06ZhxowZ2LFjB9rb27F06VIZ7RCRJFIC6K233sKff/6JjRs3wu12Y+rUqTh27NhDF6aJ6Mkm5eeABsrn88FsNkPTNF4DIhqC+voe5b1gRCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikibsADp9+jQWLFiAjIwMxMXF4ccffwwZF0Jg48aNeO655zBixAjY7Xb89ttvITV3797F4sWLoSgKUlNTsWzZMrS1tQ3oRIgo9oQdQO3t7ZgyZQp27drV6/i2bduwc+dO7NmzB+fOnUNSUhLy8vLQ0dGh1yxevBgNDQ2oqqpCZWUlTp8+jRUrVvT/LIgoNokBACAOHTqk7weDQWG1WsWnn36qH2tpaREmk0l89913QgghLl++LACI8+fP6zVHjx4VcXFx4saNG316XU3TBAChadpA2ieiQdLX92hErwFdv34dbrcbdrtdP2Y2m5GTkwOXywUAcLlcSE1NxbRp0/Qau90Og8GAc+fO9fq8fr8fPp8vZCOi2BfRAHK73QAAi8USctxisehjbrcb6enpIePx8fFIS0vTax5UVlYGs9msb5mZmZFsm4gkiYlVsNLSUmiapm/Nzc2yWyKiCIhoAFmtVgCAx+MJOe7xePQxq9UKr9cbMt7V1YW7d+/qNQ8ymUxQFCVkI6LYF9EAGjt2LKxWK5xOp37M5/Ph3LlzUFUVAKCqKlpaWlBbW6vXnDhxAsFgEDk5OZFsh4iGuPhwH9DW1oZr167p+9evX0ddXR3S0tJgs9lQXFyMjz/+GOPHj8fYsWOxYcMGZGRkYOHChQCAF198Ea+99hqWL1+OPXv2oLOzE0VFRXj77beRkZERsRMjohgQ7vLayZMnBYCHtsLCQiHEX0vxGzZsEBaLRZhMJjF37lzR2NgY8hx37twR77zzjkhOThaKooilS5eK1tbWiC/xEZEcfX2PxgkhhMT86xefzwez2QxN03g9iGgI6ut7NCZWwYjoycQAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKQJ+7/lIYq01lu/IdB2V983mdORnD5WYkcULQwgks5T78T/Xr+o7z/74n8xgJ4S/ApGRNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpAkrgMrKyjB9+nSkpKQgPT0dCxcuRGNjY0hNR0cHHA4HRo4cieTkZBQUFMDj8YTUNDU1IT8/H4mJiUhPT8e6devQ1dU18LMhopgSVgBVV1fD4XDg7NmzqKqqQmdnJ3Jzc9He3q7XrF27FocPH0ZFRQWqq6tx8+ZNLFq0SB/v7u5Gfn4+AoEAzpw5g/3792Pfvn3YuHFj5M6KiGKDGACv1ysAiOrqaiGEEC0tLSIhIUFUVFToNVeuXBEAhMvlEkIIceTIEWEwGITb7dZrysvLhaIowu/39+l1NU0TAISmaQNpn4aI346Xi5o9y/XtevU3sluiAerre3RA14A0TQMApKWlAQBqa2vR2dkJu92u10yYMAE2mw0ulwsA4HK5MGnSJFgsFr0mLy8PPp8PDQ0Nvb6O3++Hz+cL2Ygo9vU7gILBIIqLizF79mxMnDgRAOB2u2E0GpGamhpSa7FY4Ha79Zq/h0/PeM9Yb8rKymA2m/UtMzOzv20T0RDS7wByOBy4dOkSDh48GMl+elVaWgpN0/Stubl50F+TiAZfv25GLSoqQmVlJU6fPo3Ro0frx61WKwKBAFpaWkI+BXk8HlitVr2mpqYm5Pl6Vsl6ah5kMplgMpn60yoRDWFhfQISQqCoqAiHDh3CiRMnMHZs6B3L2dnZSEhIgNPp1I81NjaiqakJqqoCAFRVRX19Pbxer15TVVUFRVGQlZU1kHMhohgT1icgh8OBAwcO4KeffkJKSop+zcZsNmPEiBEwm81YtmwZSkpKkJaWBkVRsHr1aqiqipkzZwIAcnNzkZWVhSVLlmDbtm1wu91Yv349HA4HP+UQPWXCCqDy8nIAwKuvvhpyfO/evXjvvfcAANu3b4fBYEBBQQH8fj/y8vKwe/duvXbYsGGorKzEqlWroKoqkpKSUFhYiC1btgzsTIgo5sQJIYTsJsLl8/lgNpuhaRoURZHdDg3QtZ/3PPQLycb89xKJHdFA9fU9ynvBiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0oT1P6MS9Yff78f9+/cfOd7Z2flAfQAtLS2PrE9MTITRaIxUeyQRA4gG3TfffIPNmzc/cnzdmy9BnfCcvv/DDxUoX7rpkfU7duzAm2++GdEeSQ4GEA26trY23Lhx45Hj7fcm4dr9l3G3MwPPJvyB1rZ/P7b+3r17g9EmSRDWNaDy8nJMnjwZiqJAURSoqoqjR4/q4x0dHXA4HBg5ciSSk5NRUFAAj8cT8hxNTU3Iz89HYmIi0tPTsW7dOnR1dUXmbCgm/ef+VFy791cA/fteDv7oyJLdEkVJWAE0evRobN26FbW1tbhw4QLmzJmD119/HQ0NDQCAtWvX4vDhw6ioqEB1dTVu3ryJRYsW6Y/v7u5Gfn4+AoEAzpw5g/3792Pfvn3YuHFjZM+KYkpbVyp6/ikKGNDenSqzHYqisAJowYIFmD9/PsaPH4/nn38en3zyCZKTk3H27FlomoavvvoKn332GebMmYPs7Gzs3bsXZ86cwdmzZwEAP//8My5fvoxvv/0WU6dOxbx58/DRRx9h165dCAQCg3KCNPRlmP6D+Dg/AAFj3H1Yjddlt0RR0u9rQN3d3aioqEB7eztUVUVtbS06Oztht9v1mgkTJsBms8HlcmHmzJlwuVyYNGkSLBaLXpOXl4dVq1ahoaEBL730Ulg9XL16FcnJyf09BYqSB7+GP+jX/6mCufkPtHSlIy3BDfetq4+tv3nzJi5fvhzJFinC2tra+lQXdgDV19dDVVV0dHQgOTkZhw4dQlZWFurq6mA0GpGamhpSb7FY4Ha7AQButzskfHrGe8Yexe/3w+/36/s+nw8AoGkarx/FgMctwQNAdd3vAH7v8/Pdu3fvscv0JF97e3uf6sIOoBdeeAF1dXXQNA0//PADCgsLUV1dHXaD4SgrK+t1GTcnJweKogzqa9PAnTt3LqLPN27cOMyaNSuiz0mR1fMh4Z+E/ZPQRqMR48aNQ3Z2NsrKyjBlyhR8/vnnsFqtCAQe/gEyj8cDq9UKALBarQ99HO/Z76npTWlpKTRN07fm5uZw2yaiIWjAt2IEg0H4/X5kZ2cjISEBTqdTH2tsbERTUxNUVQUAqKqK+vp6eL1evaaqqgqKoiAr69FLryaTSV/679mIKPaF9RWstLQU8+bNg81mQ2trKw4cOIBTp07h+PHjMJvNWLZsGUpKSpCWlgZFUbB69WqoqoqZM2cCAHJzc5GVlYUlS5Zg27ZtcLvdWL9+PRwOB0wm06CcIBENXWEFkNfrxbvvvotbt27BbDZj8uTJOH78OP71r38BALZv3w6DwYCCggL4/X7k5eVh9+7d+uOHDRuGyspKrFq1CqqqIikpCYWFhdiyZUtkz4qGlJ5PsJHC+8CeHHFCCCG7iXD5fD6YzWZomsavYzGgtbU1oqtWaWlpSEpKitjzUeT19T3Ke8Fo0KWkpCAlJUV2GzQE8fcBEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImniZTfQH0IIAIDP55PcCRH1pue92fNefZSYDKA7d+4AADIzMyV3QkSP09raCrPZ/MjxmAygtLQ0AEBTU9NjT45C+Xw+ZGZmorm5GYqiyG4nJnDO+kcIgdbWVmRkZDy2LiYDyGD469KV2WzmP4p+UBSF8xYmzln4+vLhgBehiUgaBhARSROTAWQymbBp0yaYTCbZrcQUzlv4OGeDK0780zoZEdEgiclPQET0ZGAAEZE0DCAikoYBRETSxGQA7dq1C2PGjMHw4cORk5ODmpoa2S1JU1ZWhunTpyMlJQXp6elYuHAhGhsbQ2o6OjrgcDgwcuRIJCcno6CgAB6PJ6SmqakJ+fn5SExMRHp6OtatW4eurq5onoo0W7duRVxcHIqLi/VjnLMoETHm4MGDwmg0iq+//lo0NDSI5cuXi9TUVOHxeGS3JkVeXp7Yu3evuHTpkqirqxPz588XNptNtLW16TUrV64UmZmZwul0igsXLoiZM2eKWbNm6eNdXV1i4sSJwm63i19//VUcOXJEjBo1SpSWlso4paiqqakRY8aMEZMnTxZr1qzRj3POoiPmAmjGjBnC4XDo+93d3SIjI0OUlZVJ7Gro8Hq9AoCorq4WQgjR0tIiEhISREVFhV5z5coVAUC4XC4hhBBHjhwRBoNBuN1uvaa8vFwoiiL8fn90TyCKWltbxfjx40VVVZV45ZVX9ADinEVPTH0FCwQCqK2thd1u148ZDAbY7Xa4XC6JnQ0dmqYB+P8bdmtra9HZ2RkyZxMmTIDNZtPnzOVyYdKkSbBYLHpNXl4efD4fGhoaoth9dDkcDuTn54fMDcA5i6aYuhn19u3b6O7uDvlLBwCLxYKrV69K6mroCAaDKC4uxuzZszFx4kQAgNvthtFoRGpqakitxWKB2+3Wa3qb056xJ9HBgwdx8eJFnD9//qExzln0xFQA0eM5HA5cunQJv/zyi+xWhrTm5masWbMGVVVVGD58uOx2nmox9RVs1KhRGDZs2EOrER6PB1arVVJXQ0NRUREqKytx8uRJjB49Wj9utVoRCATQ0tISUv/3ObNarb3Oac/Yk6a2thZerxcvv/wy4uPjER8fj+rqauzcuRPx8fGwWCycsyiJqQAyGo3Izs6G0+nUjwWDQTidTqiqKrEzeYQQKCoqwqFDh3DixAmMHTs2ZDw7OxsJCQkhc9bY2IimpiZ9zlRVRX19Pbxer15TVVUFRVGQlZUVnROJorlz56K+vh51dXX6Nm3aNCxevFj/M+csSmRfBQ/XwYMHhclkEvv27ROXL18WK1asEKmpqSGrEU+TVatWCbPZLE6dOiVu3bqlb/fu3dNrVq5cKWw2mzhx4oS4cOGCUFVVqKqqj/csKefm5oq6ujpx7Ngx8eyzzz5VS8p/XwUTgnMWLTEXQEII8cUXXwibzSaMRqOYMWOGOHv2rOyWpAHQ67Z371695v79++L9998XzzzzjEhMTBRvvPGGuHXrVsjz/P7772LevHlixIgRYtSoUeKDDz4QnZ2dUT4beR4MIM5ZdPDXcRCRNDF1DYiIniwMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISJr/AxFJMCBRCGiGAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import gym\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "\n",
    "    def __init__(self):\n",
    "        env = gym.make('CartPole-v1', render_mode='rgb_array')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state, _ = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, terminated, truncated, info = self.env.step(action)\n",
    "        over = terminated or truncated\n",
    "\n",
    "        #限制最大步数\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 200:\n",
    "            over = True\n",
    "        \n",
    "        #没坚持到最后,扣分\n",
    "        if over and self.step_n < 200:\n",
    "            reward = -1000\n",
    "\n",
    "        return state, reward, over\n",
    "\n",
    "    #打印游戏图像\n",
    "    def show(self):\n",
    "        from matplotlib import pyplot as plt\n",
    "        plt.figure(figsize=(3, 3))\n",
    "        plt.imshow(self.env.render())\n",
    "        plt.show()\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()\n",
    "\n",
    "env.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ecfbe912",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Sequential(\n",
       "   (0): Linear(in_features=4, out_features=64, bias=True)\n",
       "   (1): ReLU()\n",
       "   (2): Linear(in_features=64, out_features=64, bias=True)\n",
       "   (3): ReLU()\n",
       "   (4): Linear(in_features=64, out_features=2, bias=True)\n",
       " ),\n",
       " Sequential(\n",
       "   (0): Linear(in_features=4, out_features=64, bias=True)\n",
       "   (1): ReLU()\n",
       "   (2): Linear(in_features=64, out_features=64, bias=True)\n",
       "   (3): ReLU()\n",
       "   (4): Linear(in_features=64, out_features=2, bias=True)\n",
       " ))"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "#定义模型,评估状态下每个动作的价值\n",
    "model = torch.nn.Sequential(\n",
    "    torch.nn.Linear(4, 64),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(64, 64),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(64, 2),\n",
    ")\n",
    "\n",
    "#延迟更新的模型,用于计算target\n",
    "model_delay = torch.nn.Sequential(\n",
    "    torch.nn.Linear(4, 64),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(64, 64),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(64, 2),\n",
    ")\n",
    "\n",
    "#复制参数\n",
    "model_delay.load_state_dict(model.state_dict())\n",
    "\n",
    "model, model_delay"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "84cbf0ff",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-990.0"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython import display\n",
    "import random\n",
    "\n",
    "\n",
    "#玩一局游戏并记录数据\n",
    "def play(show=False):\n",
    "    data = []\n",
    "    reward_sum = 0\n",
    "\n",
    "    state = env.reset()\n",
    "    over = False\n",
    "    while not over:\n",
    "        action = model(torch.FloatTensor(state).reshape(1, 4)).argmax().item()\n",
    "        if random.random() < 0.1:\n",
    "            action = env.action_space.sample()\n",
    "\n",
    "        next_state, reward, over = env.step(action)\n",
    "\n",
    "        data.append((state, action, reward, next_state, over))\n",
    "        reward_sum += reward\n",
    "\n",
    "        state = next_state\n",
    "\n",
    "        if show:\n",
    "            display.clear_output(wait=True)\n",
    "            env.show()\n",
    "\n",
    "    return data, reward_sum\n",
    "\n",
    "\n",
    "play()[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7bfcfd21",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_7080/2282743005.py:27: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at  ../torch/csrc/utils/tensor_new.cpp:201.)\n",
      "  state = torch.FloatTensor([i[0] for i in data]).reshape(-1, 4)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(202,\n",
       " (array([-0.02233213, -0.03595649,  0.02841122,  0.03100251], dtype=float32),\n",
       "  1,\n",
       "  1.0,\n",
       "  array([-0.02305126,  0.15874675,  0.02903127, -0.2525827 ], dtype=float32),\n",
       "  False))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#数据池\n",
    "class Pool:\n",
    "\n",
    "    def __init__(self):\n",
    "        self.pool = []\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.pool)\n",
    "\n",
    "    def __getitem__(self, i):\n",
    "        return self.pool[i]\n",
    "\n",
    "    #更新动作池\n",
    "    def update(self):\n",
    "        #每次更新不少于N条新数据\n",
    "        old_len = len(self.pool)\n",
    "        while len(pool) - old_len < 200:\n",
    "            self.pool.extend(play()[0])\n",
    "\n",
    "        #只保留最新的N条数据\n",
    "        self.pool = self.pool[-2_0000:]\n",
    "\n",
    "    #获取一批数据样本\n",
    "    def sample(self):\n",
    "        data = random.sample(self.pool, 64)\n",
    "\n",
    "        state = torch.FloatTensor([i[0] for i in data]).reshape(-1, 4)\n",
    "        action = torch.LongTensor([i[1] for i in data]).reshape(-1, 1)\n",
    "        reward = torch.FloatTensor([i[2] for i in data]).reshape(-1, 1)\n",
    "        next_state = torch.FloatTensor([i[3] for i in data]).reshape(-1, 4)\n",
    "        over = torch.LongTensor([i[4] for i in data]).reshape(-1, 1)\n",
    "\n",
    "        return state, action, reward, next_state, over\n",
    "\n",
    "\n",
    "pool = Pool()\n",
    "pool.update()\n",
    "pool.sample()\n",
    "\n",
    "len(pool), pool[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "018c4b1e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 410 -946.4\n",
      "100 20000 -2.55\n",
      "200 20000 200.0\n",
      "300 20000 200.0\n",
      "400 20000 141.55\n",
      "500 20000 -102.55\n",
      "600 20000 200.0\n",
      "700 20000 200.0\n",
      "800 20000 200.0\n",
      "900 20000 49.3\n"
     ]
    }
   ],
   "source": [
    "#训练\n",
    "def train():\n",
    "    model.train()\n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=2e-4)\n",
    "    loss_fn = torch.nn.MSELoss()\n",
    "\n",
    "    #共更新N轮数据\n",
    "    for epoch in range(1000):\n",
    "        pool.update()\n",
    "\n",
    "        #每次更新数据后,训练N次\n",
    "        for i in range(200):\n",
    "\n",
    "            #采样N条数据\n",
    "            state, action, reward, next_state, over = pool.sample()\n",
    "\n",
    "            #计算value\n",
    "            value = model(state).gather(dim=1, index=action)\n",
    "\n",
    "            #计算target\n",
    "            with torch.no_grad():\n",
    "                #使用原模型计算动作,使用延迟模型计算target,进一步缓解自举\n",
    "                next_action = model(next_state).argmax(dim=1, keepdim=True)\n",
    "                target = model_delay(next_state).gather(dim=1,\n",
    "                                                        index=next_action)\n",
    "            target = target * 0.99 * (1 - over) + reward\n",
    "\n",
    "            loss = loss_fn(value, target)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            optimizer.zero_grad()\n",
    "\n",
    "        #复制参数\n",
    "        if (epoch + 1) % 5 == 0:\n",
    "            model_delay.load_state_dict(model.state_dict())\n",
    "\n",
    "        if epoch % 100 == 0:\n",
    "            test_result = sum([play()[-1] for _ in range(20)]) / 20\n",
    "            print(epoch, len(pool), test_result)\n",
    "\n",
    "\n",
    "train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a101d0e6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAS7UlEQVR4nO3df2xTdb8H8Pe6roWxnc5trnXZelkiESa/dMB25D5Xg5WJCxHZH2oITkIwYkeEGaJLZIo/MoO5QfGB8Y8CyROcmQa9TFDngBEvHYPhcseAXY16twhtBezpNli7td/7x5Odx8qG6xj9rvB+JSex3++np5/zxfOmPWcdCUIIASIiCQyyGyCi2xcDiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpJEWQNu3b8fUqVMxadIkFBYWoqWlRVYrRCSJlAD65JNPUFFRgddeew2nTp3CnDlzUFxcDK/XK6MdIpIkQcaXUQsLCzF//nz8/e9/BwCEw2Hk5uZi3bp1eOWVV2LdDhFJYoz1CwaDQbS2tqKyslIfMxgMcDgccLlcwz4nEAggEAjoj8PhMC5fvoyMjAwkJCTc9J6JKDpCCPT09CA7OxsGw8gftGIeQBcvXkQoFILVao0Yt1qtOHfu3LDPqa6uxubNm2PRHhGNo+7ubuTk5Iw4H/MAGovKykpUVFTojzVNg91uR3d3NxRFkdgZEQ3H7/cjNzcXqamp162LeQBlZmYiMTERHo8nYtzj8cBmsw37HLPZDLPZfM24oigMIKIJ7K8ukcT8LpjJZEJBQQEaGxv1sXA4jMbGRqiqGut2iEgiKR/BKioqUFZWhnnz5mHBggV477330NfXh1WrVsloh4gkkRJATz75JH777TdUVVXB7XZj7ty5+Oqrr665ME1EtzYpPwd0o/x+PywWCzRN4zUgoglotOcovwtGRNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaaIOoKNHj2Lp0qXIzs5GQkICPv/884h5IQSqqqpw1113YfLkyXA4HPjhhx8iai5fvowVK1ZAURSkpaVh9erV6O3tvaEDIaL4E3UA9fX1Yc6cOdi+ffuw81u2bMG2bduwc+dOHD9+HFOmTEFxcTH6+/v1mhUrVqCjowMNDQ2or6/H0aNH8dxzz439KIgoPokbAEDs27dPfxwOh4XNZhPvvvuuPubz+YTZbBYff/yxEEKIM2fOCADixIkTes3BgwdFQkKC+PXXX0f1upqmCQBC07QbaZ+IbpLRnqPjeg3o559/htvthsPh0McsFgsKCwvhcrkAAC6XC2lpaZg3b55e43A4YDAYcPz48WH3GwgE4Pf7IzYiin/jGkButxsAYLVaI8atVqs+53a7kZWVFTFvNBqRnp6u1/xZdXU1LBaLvuXm5o5n20QkSVzcBausrISmafrW3d0tuyUiGgfjGkA2mw0A4PF4IsY9Ho8+Z7PZ4PV6I+YHBwdx+fJlvebPzGYzFEWJ2Igo/o1rAOXl5cFms6GxsVEf8/v9OH78OFRVBQCoqgqfz4fW1la95tChQwiHwygsLBzPdohogjNG+4Te3l78+OOP+uOff/4ZbW1tSE9Ph91ux/r16/HWW29h2rRpyMvLw6ZNm5CdnY1ly5YBAGbMmIFHH30Ua9aswc6dOzEwMIDy8nI89dRTyM7OHrcDI6I4EO3ttcOHDwsA12xlZWVCiH/eit+0aZOwWq3CbDaLhx9+WHR2dkbs49KlS+Lpp58WKSkpQlEUsWrVKtHT0zPut/iISI7RnqMJQgghMf/GxO/3w2KxQNM0Xg8imoBGe47GxV0wIro1MYCISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaaL+Z3mIJjIRDkPrPo1Q8Ko+NiUrD5MsWdd5FsnCAKJbihAhdDd/in6fWx/7t7+tYABNUPwIRkTSMICISBoGEBFJwwAiImkYQEQkDQOIiKSJKoCqq6sxf/58pKamIisrC8uWLUNnZ2dETX9/P5xOJzIyMpCSkoLS0lJ4PJ6Imq6uLpSUlCA5ORlZWVnYuHEjBgcHb/xoiCiuRBVATU1NcDqdaG5uRkNDAwYGBrB48WL09fXpNRs2bMD+/ftRV1eHpqYmnD9/HsuXL9fnQ6EQSkpKEAwGcezYMezZswe7d+9GVVXV+B0VEcUHcQO8Xq8AIJqamoQQQvh8PpGUlCTq6ur0mrNnzwoAwuVyCSGEOHDggDAYDMLtdus1NTU1QlEUEQgERvW6mqYJAELTtBtpn25BocGg+J/aTaJl5xp983Qckd3WbWe05+gNXQPSNA0AkJ6eDgBobW3FwMAAHA6HXjN9+nTY7Xa4XC4AgMvlwqxZs2C1WvWa4uJi+P1+dHR0DPs6gUAAfr8/YiOi+DfmAAqHw1i/fj0WLlyImTNnAgDcbjdMJhPS0tIiaq1WK9xut17zx/AZmh+aG051dTUsFou+5ebmjrVtIppAxhxATqcTp0+fRm1t7Xj2M6zKykpomqZv3d3dN/01iejmG9OXUcvLy1FfX4+jR48iJydHH7fZbAgGg/D5fBHvgjweD2w2m17T0tISsb+hu2RDNX9mNpthNpvH0ioRTWBRvQMSQqC8vBz79u3DoUOHkJeXFzFfUFCApKQkNDY26mOdnZ3o6uqCqqoAAFVV0d7eDq/Xq9c0NDRAURTk5+ffyLEQUZyJ6h2Q0+nE3r178cUXXyA1NVW/ZmOxWDB58mRYLBasXr0aFRUVSE9Ph6IoWLduHVRVRVFREQBg8eLFyM/Px8qVK7Flyxa43W68+uqrcDqdfJdDdJuJKoBqamoAAA899FDE+K5du/Dss88CALZu3QqDwYDS0lIEAgEUFxdjx44dem1iYiLq6+uxdu1aqKqKKVOmoKysDG+88caNHQkRxZ0EIYSQ3US0/H4/LBYLNE2Doiiy26EJJBwaQMenb17zC8my8h+U2NXtZ7TnKL8LRkTSMICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikiaqfxmVaCK4cuUKgsHgsHMiNIhwOBwxdvXqVfh8vhH3l5qaisTExPFskUaJAURxp7KyEp999tmwc0mJBvznmn9HTmaKPrZ582Z8c6p72Hqj0Ygvv/wS9957703pla6PAURx5/fff8evv/467FyS0YArQSPO9C1EX8iC3Enn4PN9N2K90WjEwMDAzWyXriOqa0A1NTWYPXs2FEWBoihQVRUHDx7U5/v7++F0OpGRkYGUlBSUlpbC4/FE7KOrqwslJSVITk5GVlYWNm7ciMHBwfE5GrrtCWHA6d6/oas/H5cGctDe+x+4GMyR3RaNIKoAysnJwTvvvIPW1lacPHkSixYtwuOPP46Ojg4AwIYNG7B//37U1dWhqakJ58+fx/Lly/Xnh0IhlJSUIBgM4tixY9izZw92796Nqqqq8T0qum0JJKAvlAYgAQAQEiZcDadK7YlGFtVHsKVLl0Y8fvvtt1FTU4Pm5mbk5OTgww8/xN69e7Fo0SIAwK5duzBjxgw0NzejqKgI33zzDc6cOYNvv/0WVqsVc+fOxZtvvomXX34Zr7/+Okwm0/gdGd2WDAlhZJt/QAi5CCMRKYk+3JF0QXZbNIIxXwMKhUKoq6tDX18fVFVFa2srBgYG4HA49Jrp06fDbrfD5XKhqKgILpcLs2bNgtVq1WuKi4uxdu1adHR04L777ouqh3PnziElJeWvC+mWcr07WqFwGM3H/oEBUzOuhBRkmbrwf13/O2K9EAI//fQT//IbZ729vaOqizqA2tvboaoq+vv7kZKSgn379iE/Px9tbW0wmUxIS0uLqLdarXC73QAAt9sdET5D80NzIwkEAggEAvpjv98PANA0jdePbkPXu2gcDgv813+fBXB21Pvr6em5bqhR9Pr6+kZVF3UA3XPPPWhra4Omafj0009RVlaGpqamqBuMRnV1NTZv3nzNeGFhIRRFuamvTRPPnXfeOW77SkhIwJw5czB37txx2yf9603CX4n6J6FNJhPuvvtuFBQUoLq6GnPmzMH7778Pm82GYDB4zd8kHo8HNpsNAGCz2a65Kzb0eKhmOJWVldA0Td+6u4f/mQ4iii83/FWMcDiMQCCAgoICJCUlobGxUZ/r7OxEV1cXVFUFAKiqivb2dni9Xr2moaEBiqIgPz9/xNcwm836rf+hjYjiX1QfwSorK7FkyRLY7Xb09PRg7969OHLkCL7++mtYLBasXr0aFRUVSE9Ph6IoWLduHVRVRVFREQBg8eLFyM/Px8qVK7Flyxa43W68+uqrcDqdMJvNN+UAiWjiiiqAvF4vnnnmGVy4cAEWiwWzZ8/G119/jUceeQQAsHXrVhgMBpSWliIQCKC4uBg7duzQn5+YmIj6+nqsXbsWqqpiypQpKCsrwxtvvDG+R0W3tMmTJ4/bu2Cj0cjvgUmUIIQQspuIlt/vh8VigaZp/Dh2G7p06RKuXLkybvuzWq28DT/ORnuO8rtgFHcyMjKQkZEhuw0aB/x9QEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEgao+wGxkIIAQDw+/2SOyGi4Qydm0Pn6kjiMoAuXboEAMjNzZXcCRFdT09PDywWy4jzcRlA6enpAICurq7rHhxF8vv9yM3NRXd3NxRFkd1OXOCajY0QAj09PcjOzr5uXVwGkMHwz0tXFouF/1OMgaIoXLcocc2iN5o3B7wITUTSMICISJq4DCCz2YzXXnsNZrNZditxhesWPa7ZzZUg/uo+GRHRTRKX74CI6NbAACIiaRhARCQNA4iIpInLANq+fTumTp2KSZMmobCwEC0tLbJbkqa6uhrz589HamoqsrKysGzZMnR2dkbU9Pf3w+l0IiMjAykpKSgtLYXH44mo6erqQklJCZKTk5GVlYWNGzdicHAwlocizTvvvIOEhASsX79eH+OaxYiIM7W1tcJkMomPPvpIdHR0iDVr1oi0tDTh8XhktyZFcXGx2LVrlzh9+rRoa2sTjz32mLDb7aK3t1evef7550Vubq5obGwUJ0+eFEVFReKBBx7Q5wcHB8XMmTOFw+EQ33//vThw4IDIzMwUlZWVMg4pplpaWsTUqVPF7NmzxYsvvqiPc81iI+4CaMGCBcLpdOqPQ6GQyM7OFtXV1RK7mji8Xq8AIJqamoQQQvh8PpGUlCTq6ur0mrNnzwoAwuVyCSGEOHDggDAYDMLtdus1NTU1QlEUEQgEYnsAMdTT0yOmTZsmGhoaxIMPPqgHENcsduLqI1gwGERrayscDoc+ZjAY4HA44HK5JHY2cWiaBuBfX9htbW3FwMBAxJpNnz4ddrtdXzOXy4VZs2bBarXqNcXFxfD7/ejo6Ihh97HldDpRUlISsTYA1yyW4urLqBcvXkQoFIr4QwcAq9WKc+fOSepq4giHw1i/fj0WLlyImTNnAgDcbjdMJhPS0tIiaq1WK9xut14z3JoOzd2KamtrcerUKZw4ceKaOa5Z7MRVANH1OZ1OnD59Gt99953sVia07u5uvPjii2hoaMCkSZNkt3Nbi6uPYJmZmUhMTLzmboTH44HNZpPU1cRQXl6O+vp6HD58GDk5Ofq4zWZDMBiEz+eLqP/jmtlstmHXdGjuVtPa2gqv14v7778fRqMRRqMRTU1N2LZtG4xGI6xWK9csRuIqgEwmEwoKCtDY2KiPhcNhNDY2QlVViZ3JI4RAeXk59u3bh0OHDiEvLy9ivqCgAElJSRFr1tnZia6uLn3NVFVFe3s7vF6vXtPQ0ABFUZCfnx+bA4mhhx9+GO3t7Whra9O3efPmYcWKFfp/c81iRPZV8GjV1tYKs9ksdu/eLc6cOSOee+45kZaWFnE34naydu1aYbFYxJEjR8SFCxf07cqVK3rN888/L+x2uzh06JA4efKkUFVVqKqqzw/dUl68eLFoa2sTX331lbjzzjtvq1vKf7wLJgTXLFbiLoCEEOKDDz4QdrtdmEwmsWDBAtHc3Cy7JWkADLvt2rVLr7l69ap44YUXxB133CGSk5PFE088IS5cuBCxn19++UUsWbJETJ48WWRmZoqXXnpJDAwMxPho5PlzAHHNYoO/joOIpImra0BEdGthABGRNAwgIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNL8P4vPP8yaVGARAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "200.0"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "play(True)[-1]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:pt39]",
   "language": "python",
   "name": "conda-env-pt39-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
